/** 
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package com.tompai.datagather.tencent;

import java.text.ParseException;
import java.time.Clock;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;

import com.google.gson.Gson;
import com.tompai.common.time.TimeTransUtils;
import com.tompai.datagather.common.MybatisUtil;
import com.tompai.datagather.dao.TencentNewsMapper;
import com.tompai.datagather.model.CrawlDatum;
import com.tompai.datagather.model.CrawlDatums;
import com.tompai.datagather.model.Page;
import com.tompai.datagather.model.TenNewsJSonBean;
import com.tompai.datagather.model.TenNewsJSonBean.Data.Article_info;
import com.tompai.datagather.model.TencentNewsWithBLOBs;
import com.tompai.datagather.net.HttpRequest;
import com.tompai.datagather.plugin.berkeley.BreadthCrawler;

/**
 * @desc: datagather-tools
 * @name: TenNewsCrawler.java
 * @author: tompai
 * @createTime: 2018年4月28日 下午9:25:50
 * @history:
 * @version: v1.0
 */

public class TenNewsCrawler extends BreadthCrawler {

	private static final Logger logger = Logger.getLogger(TenNewsCrawler.class);
	String taskId;
	
	public TenNewsCrawler(String crawlPath, boolean autoParse) {
		super(crawlPath, autoParse);
		//Clock clock = Clock.systemDefaultZone();// 获取系统默认时区 (当前瞬时时间 )
		//long currentTime = clock.millis();
		// long begainPoint=1525191474487L;
		//LocalDate date = LocalDate.now();

		/*String date="2018-05-16";
		String currentTime="1526440358708";
		//logger.info("current date:"+date);
		TenNewsCrawler crawler = new TenNewsCrawler("./temp/json_crawler", true);
		String url = "http://roll.news.qq.com/interface/cpcroll.php?callback=rollback&site=news&mode=1&cata=&date="
				+ date + "&page=1&_=" + currentTime;
		logger.info("current URL:"+url);
		crawler.addSeed(new CrawlDatum(url)
				.meta("method", "GET")
				.meta("outputData", "id=a")
				.meta("url", url)
				.type("rollnews")
				.meta("depth", 1));*/
	}
	
	@Override
	public Page getResponse(CrawlDatum crawlDatum) throws Exception {
		
		HttpRequest request = new HttpRequest(crawlDatum.url());

		request.setMethod(crawlDatum.meta("method"));
		String outputData = crawlDatum.meta("outputData");
		if (outputData != null) {
			request.setOutputData(outputData.getBytes("utf-8"));
		}
		// 通过下面方式可以设置Cookie、User-Agent等http请求头信息
		request.setCookie("ad_play_index=55; roll_mod=1; pgv_info=ssid=s553581536; ts_last=news.qq.com/articleList/rolls/; ts_refer=www.google.co.jp/; pgv_pvid=851153760; ts_uid=4454223780");
		request.setUserAgent(
				"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36");
		request.addHeader("Host", "roll.news.qq.com");
		request.addHeader("Accept", "*/*");
		request.addHeader("Referer", "ttp://news.qq.com/articleList/rolls/");
		request.addHeader("Accept-Language", "zh-CN,zh;q=0.9");
		request.addHeader("Accept-Encoding", "gzip, deflate");

		return request.responsePage();

	}

	@Override
	public void visit(Page page, CrawlDatums next) {

		logger.info("Url: "+page.url());
		String formatType = "yyyy-MM-dd HH:mm:ss";

		String inputLine = page.html();
		// logger.info("HTML信息：" + inputLine);
		String regex = "rollback\\((.*?)\\)";

		TenNewsJSonBean tBean = new TenNewsJSonBean();
		Gson gson = new Gson();

		Pattern r = Pattern.compile(regex);
		Matcher m = r.matcher(inputLine);
		if (!m.matches()) {
			throw new IllegalArgumentException("Bad Input");
		}
		// 分组0得到的是整个原字符串    //
		// =================== //
		TencentNewsMapper newsMapper;
		newsMapper = MybatisUtil.getSession().getMapper(TencentNewsMapper.class);
		TencentNewsWithBLOBs tenews = new TencentNewsWithBLOBs();
		try {
			for (int i = 1; i < m.groupCount() + 1; i++) {
				tBean = gson.fromJson(m.group(i), TenNewsJSonBean.class);
				List<Article_info> info = tBean.getData().getArticle_info();
				for (int index = 0; index < info.size(); index++) {

					String title = info.get(index).getTitle();
					String url = info.get(index).getUrl();
					logger.info("title:" + title + "\turl:" + url);
					String time = info.get(index).getTime();
					String articalColumn = info.get(index).getColumn();
					tenews.setOriginalJson(inputLine);

					tenews.setPostUrl(page.url());
					tenews.setTitle(title);
					tenews.setOriginalSource("腾讯新闻");
					tenews.setTitleUrl(url);
					// =================================================
					if (!articalColumn.equals("图片")) {
						TenArticalTask articalCrawler = new TenArticalTask();

						articalCrawler.setUrl(url);
						articalCrawler.run();

						tenews.setArticalContent(articalCrawler.getContent());
						tenews.setAuthor(articalCrawler.getAuthor());
						tenews.setResponsEditor(articalCrawler.getResponsEditor());
					}
					// ===========================================
					tenews.setArticalColumn(articalColumn);
					tenews.setPublishTime(TimeTransUtils.stringToDate(time, formatType));

					// get system current date//
					LocalDateTime datetime = LocalDateTime.now();
					tenews.setCollectTime(TimeTransUtils.localDateTimeToDate(datetime));
					// =============================
					newsMapper.insert(tenews);
					// =============================
					// commit data//
					MybatisUtil.getSession().commit();
				}
			}

		} catch (ParseException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			LOG.error(e.getMessage());
		}
	}

	/**
	 *
	 * @param args
	 *            参数
	 * @throws Exception
	 *             异常
	 */
	public static void main(String[] args) throws Exception {

		do {
			TenNewsCrawler crawler = new TenNewsCrawler("./temp/json_crawler", true);
			crawler.start(1);
			Thread.sleep(3 * 60 * 1000);
			logger.info("will wait 10 minutes!");
		} while (true);

	}
}
